#
# usage:
# ... <titles_with_ids num_ratings max_duplicates
#
# titles_with_ids: file like small.titles.new.with_ids
# num_ratings: file like data/num_movie_ratings_training_set.txt
# max_duplicates: keep no more than this many movies per title (default 1)
#

raise "see usage" if ARGV.size < 1
ratings_count_file, max_duplicates = *ARGV
if max_duplicates
  max_duplicates = max_duplicates.to_i
else
  max_duplicates = 1
end

$stderr.puts "using ratings counts from #{ratings_count_file}"
$stderr.puts "keeping at most #{max_duplicates} duplicates"

# Take small.titles.with_ids on stdin.
ls = $stdin.readlines.map {|l|
  l =~ /(\d+) (.+)/ or raise "invalid format #{l}"
  [$2.chomp, $1.to_i]
}

# Hash to find unique titles; keep list of matching ids.
hs = Hash.new {|h,k| h[k] = []}
ls.each do |title, id|
  hs[title] << id
end

# When there are duplicates, keep the title(s) with the most ratings. 
rc = Hash[File.open(ratings_count_file) {|f| f.readlines}.map {|l|
  l.split(' ').map{|x| x.to_i}}]

# Forget about duplicates.
# TODO should remove the "|| 0"...
hs.each do |title,ids|
  next if ids.size <= max_duplicates
  hs[title] = ids.sort_by{|id| rc[id] || 0}.reverse.take(max_duplicates)
end

# Spit out in same format, ordered by title number.
puts hs.map{|t,ns| ns.map{|n| [n,t]}}.
  flatten(1).
  sort_by{|n,t| n}.
  map{|n,t| "#{n} #{t}"}

# Copyright (c) 2009 John Lees-Miller
# 
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
# 
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
# 
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.

